import requests
from lxml import etree
import json

class Lagou(object):
    def __init__(self):
        self.url = "https://www.lagou.com/zhaopin/Python/"
        self.headers = {
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36"
        }
        self.url_list = [self.url+str(i)+'/' for i in range(1,6)]

    def get_data(self):
        response = requests.get(self.url,headers=self.headers)
        return response.content.decode()

    def parse_data(self,str_data):
        html = etree.HTML(str_data)
        node_list = html.xpath('//*[@id="s_position_list"]/ul/li')
        # print(node_list)
        data_list = []
        for node in node_list:
            temp = {}
            try:
                temp["岗位"] = node.xpath('./div[1]/div[1]/div[1]/a/h3/text()')[0]
                temp["薪资"] = node.xpath('./div[1]/div[1]/div[2]/div/span/text()')[0]
                temp["工作年限"] = node.xpath('./div[1]/div[1]/div[2]/div/text()')[0]
            except:
                temp["岗位"] = None
                temp["薪资"] = None
                temp["工作年限"] = None
            data_list.append(temp)
            # print(data_list)
        return data_list

    def save_data(self,data_list):
        with open("lagou.json","a")as f:
            for data in data_list:
                str_data = json.dumps(data,ensure_ascii=False)+',\n'
                f.write(str_data)

    def run(self):
        # 遍历url列表
        for url in self.url_list:
            # 发送给请求，获取响应
            str_data = self.get_data()
            # 解析数据，返回数据列表
            data_list = self.parse_data(str_data)
            # 将解析后的数据保存到文件
            self.save_data(data_list)


if __name__ == '__main__':
    lagou = Lagou()
    lagou.run()
